{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "随机梯度下降算法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "m=100000\n",
    "x=np.random.normal(size=m)\n",
    "X=x.reshape(-1,1)\n",
    "y=4.*x+3+np.random.normal(0,3,size=m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def J(theta,X_b,y):\n",
    "    try:\n",
    "        return np.sum((y-X_b.dot(theta))**2)/len(y)\n",
    "    except:\n",
    "        return float(\"inf\")\n",
    "def dJ(theta,X_b,y):\n",
    "    return X_b.T.dot(X_b.dot(theta)-y)*2/len(y)\n",
    "def gradient_descent(X_b,y,initial_theta,eta,n_inters=1e4,epsilon=1e-8):\n",
    "    theta=initial_theta\n",
    "    cur_iter=0\n",
    "    while cur_iter<n_inters:\n",
    "        gradient=dJ(theta,X_b,y)\n",
    "        last_theta=theta\n",
    "        theta=theta-eta*gradient\n",
    "        if (abs(J(theta,X_b,y)-J(last_theta,X_b,y))<epsilon):\n",
    "            break\n",
    "        cur_iter=cur_iter+1\n",
    "    return theta"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3.0021426  4.00744831]\n",
      "CPU times: total: 766 ms\n",
      "Wall time: 441 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "X_b=np.hstack([np.ones((len(X),1)),X])\n",
    "initial_theta=np.zeros(X_b.shape[1])\n",
    "eta=0.01\n",
    "theta=gradient_descent(X_b,y,initial_theta,eta)\n",
    "print(theta)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "随机梯度下降"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def dj_sgd(theta,X_b_i,y_i):\n",
    "    return X_b_i.T.dot(X_b_i.dot(theta)-y_i)*2#选取X_b中的一行数据，和对应的y的值（1个值）\n",
    "def sgd(X_b,y,initial_theta,n_iters):\n",
    "    \n",
    "    t0=5\n",
    "    t1=50\n",
    "    \"\"\"学习率计算公式：η=t0/(i_iters+t1)\"\"\"\n",
    "    def learning_rate(t):\n",
    "        return t0/(t+t1)\n",
    "    theta=initial_theta\n",
    "    for cur_iter in range(n_iters):\n",
    "        rand_i=np.random.randint(len(X_b))#X_b中任意一行数据的索引\n",
    "        gradient=dj_sgd(theta,X_b[rand_i],y[rand_i])#随机取一行数据计算梯度\n",
    "        theta=theta-learning_rate(cur_iter)*gradient#用当前η和梯度迭代一步θ\n",
    "    return theta\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: total: 344 ms\n",
      "Wall time: 346 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "X_b=np.hstack([np.ones((len(X),1)),X])#将1与X拼起来形成X_b矩阵\n",
    "initial_theta=np.zeros(X_b.shape[1])#初始化参数个数\n",
    "theta=sgd(X_b,y,initial_theta,len(X_b)//3)#训练次数为总样本数的1/3（展示随机梯度下降的强大）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
